package com.eascs.spider.ant.worker.spider;

import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.alibaba.fastjson.JSON;
import com.eascs.spider.ant.common.proj.Platform;
import com.eascs.spider.ant.common.proj.Task;
import com.eascs.spider.ant.common.proj.business.Category;
import com.eascs.spider.ant.common.util.CommonUtil;

/**
 * 新农商网 价格行情
 * @author hzp
 *
 */
public class NcPriceSpider extends AnotherSpider{

	public NcPriceSpider(Task task) {
		super(task);
	}

	@Override
	public void run(){
		//价格行情主页：http://nc.mofcom.gov.cn/channel/jghq2017/index.shtml
		//如果是主页,提取分类组，并构造新的task
		if(task.getHomepage().equals(Task.HOMEPAGE_YES)){
			crawlerHomepage();
		}else{
			crawlerPrice();
		}
	}
	
	//处理非主页任务
	private void crawlerPrice(){
		try {
			Map<String, String> header = task.getHeaderMap();
			Map<String, String> param = task.getParamMap();
			
			String html = downloader.setHeader(header)
									.setParam(param)
									.setHttpMethod(task.getMethod())
									.setUrl(task.getUrl())
									.download()
									.getHtml(); 
			
			if(Jsoup.parse(html).selectFirst("table[class=table-01 mt30]") == null){
				manager.addSendQueue(task);
				return;
			}
			if(etl(html))
				turnpage();
		} catch (Exception e) {
			manager.addSendQueue(task);
			logger.error("crawlerPrice发生异常,task:{}" , JSON.toJSONString(task) ,e);
		}
	}
	
	//处理主页的爬取请求
	private void crawlerHomepage(){
		String html = downloader.setHttpMethod("GET")
								.setUrl(task.getUrl())
								.download()
								.getHtml(); 
		List<Category> list = extractCategory(html);
		
		//为每个分类构造task对象
		String url = "http://nc.mofcom.gov.cn/channel/jghq2017/price_list.shtml";
		Map<String, String> header = new HashMap<>();
		header.put("Referer", "http://nc.mofcom.gov.cn/channel/jghq2017/price_list.shtml");
		header.put("Host", "nc.mofcom.gov.cn");
		Map<String, String> param = new HashMap<>();
		param.put("par_craft_index", "");
		param.put("p_index", "");
		SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
		String today = sdf.format(new Date(System.currentTimeMillis()));
		//String yesterday = sdf.format(new Date(System.currentTimeMillis() - 1000*3600*24));
		param.put("startTime", today);//默认取今天日期,新农商网当天价格更新的时间大约在下午四点左右
		param.put("endTime", today);//默认取今天日期
		
		for(Category cate : list){
			param.put("par_craft_index", cate.getGroupId());//该农产品所属分组id
			param.put("craft_index", cate.getSelfId());//农产品id
			Task taskTemp = new Task();
			taskTemp.setUrl(url);
			taskTemp.setMethod(Task.HTTP_GET);
			taskTemp.setHomepage(Task.HOMEPAGE_NO);//非主页任务
			taskTemp.setParamMap(param);
			taskTemp.setHeaderMap(header);
			manager.addSendQueue(taskTemp);
		}	
	}
	
	//找出分类，蔬菜-黄瓜：13075-20410
	private List<Category> extractCategory(String html){
		String regex = "<a href=/channel/jghq2017/price_list.shtml\\?craft_index=(.*?)&par_craft_index=(.*?)>(.*?)</a>";
		Matcher matcher = Pattern.compile(regex).matcher(html);
		List<Category> list = new LinkedList<>();
		Set<String> duplicate = new HashSet<>();
		while(matcher.find()){
			if(duplicate.contains(matcher.group(1))){
				continue;
			}
			duplicate.add(matcher.group(1));
			Category cate = new Category();
			cate.setGroupId(matcher.group(2));
			cate.setSelfId(matcher.group(1));
			cate.setSelfName(matcher.group(3));
			list.add(cate);
		}
		return list;
	}
	
	private void turnpage() throws ParseException{
		String url = "http://nc.mofcom.gov.cn/channel/jghq2017/price_list.shtml";
		String html = downloader.getHtml();
		Matcher matcher = Pattern.compile("var v_PageCount = (.*?);").matcher(html);
		int pages = 0;
		if(matcher.find()){
			pages = Integer.valueOf(matcher.group(1));
		}else
			return;
		for(int i=2;i<pages;i++){
			Map<String, String> header = downloader.getHeader();
			Map<String, String> param = downloader.getParam();
			
			//此时的downloader.url是上次downloader下载页面执行get请求时拼接好的url，用来做这次请求的referer
			header.put("Referer", downloader.getUrl());
			param.put("page", i+"");
			html = downloader.setHeader(header)
							 .setParam(param)
							 .setUrl(url)
							 .setHttpMethod(Task.HTTP_GET)
							 .download()
							 .getHtml();
			
			if(Jsoup.parse(html).selectFirst("table[class=table-01 mt30]") == null)//table没取到可能是网络原因导致
				continue;
			if(!etl(html))//返回false表示无数据,中断翻页
				break;;
			CommonUtil.sleep(2000);
		}
	}
	
	//返回false表示无数据
	public boolean etl(String html) throws ParseException{
		Document document = Jsoup.parse(html);
		Element table = document.selectFirst("table[class=table-01 mt30]");
		return extractPriceFromTable(table);
	}
	
	//从table中提取价格信息,返回false表示无数据
	private boolean extractPriceFromTable(Element table) throws ParseException{
		SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
		Elements trs = table.select("tr");
		if(trs.size() ==1)
			return false;
		Elements ths = trs.get(0).select("th");
		for(int i=1;i<trs.size();i++){
			Map<String,Object> map = new HashMap<String, Object>();
			Elements tdsTemp = trs.get(i).select("td");
			for(int j=0;j<ths.size();j++){
				String name =ths.get(j).text();
				String value = j==ths.size()-1?"http://nc.mofcom.gov.cn"+tdsTemp.get(j).selectFirst("a").attr("href"):tdsTemp.get(j).text();
				if(name.equals("日期")){
					map.put(name, sdf.parse(value));
				}else{
					map.put(name, value);
				}
			}
			map.put("crawler_time", new Date(System.currentTimeMillis()));
			logger.info("新农商网价格行情,单条数据,data:{}" ,map.toString());
			mongoSession.setData(map, Platform.NC_PRICE);
		}
		return true;
	}
	
}
